package com.chance.cc.crawler.development.scripts.taobao;

import com.alibaba.fastjson.JSON;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.downloader.HttpRequest;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import com.chance.cc.crawler.core.selector.Json;
import com.chance.cc.crawler.core.tags.KVTag;
import com.chance.cc.crawler.core.tags.crawler.CrawlerBusinessTags;
import com.chance.cc.crawler.meta.core.bean.crawler.CrawlerDomainUrls;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.http.NameValuePair;
import org.apache.http.client.utils.URLEncodedUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.nio.charset.Charset;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;

import static com.chance.cc.crawler.development.scripts.allfeild.AICCommonField.Tag_Field_Domain_Result_Json;
import static com.chance.cc.crawler.development.scripts.allfeild.AICCommonField.Tag_Field_ECommerce_List_Content;

/**
 * @author bx
 * @date 2020/11/13 0013 15:41
 */
public class TaobaoCommentCrawlerScript extends CrawlerCommonScript {

    private Logger log = LoggerFactory.getLogger(TaobaoCommentCrawlerScript.class);

//    private static List<String> cookies = new ArrayList<>();

    private List<String> userAgents = new ArrayList<>();

    public static final String commentUrl = "https://rate.taobao.com/feedRateList.htm?auctionNumId=%s&userNumId=%s&currentPageNum=%s&pageSize=20&folded=0,1&orderType=feedbackdate&callback=jsonp_tbcrate_reviews_list";
    public static final String defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36";
    public static final String defaultCookie = "x5sec=7b22726174656d616e616765723b32223a223538333339376234623839613038363761366666656439313535656438643037434f2f4879503046454f364e343943476d644c736b77453d227d";

    private AtomicInteger userAgentIndex = new AtomicInteger(0);
    private Object lock = new Object();


    @Override
    public List<CrawlerRecord> prepareRequest(CrawlerRequestRecord requestRecord, List<CrawlerRequestRecord> supportSourceRecords) {

        if(supportSourceRecords !=null && supportSourceRecords.size()  > 0){
            for (CrawlerRequestRecord supportSourceRecord : supportSourceRecords) {
                if (supportSourceRecord.getHttpRequest().getUrl().contains("fake-useragent.herokuapp.com/browsers")){
                    initUserAgents(supportSourceRecord); //初始user agents
                }
            }
        }

        HttpRequest httpRequest = requestRecord.getHttpRequest();
        String userAgent = getOneUserAgent();
        if(StringUtils.isBlank(userAgent)){
            userAgent = defaultUserAgent;
        }

        if(requestRecord.tagsCreator().bizTags().hasKVTag(Tag_Field_Domain_Result_Json)){
            KVTag domainResultJson = requestRecord.tagsCreator().bizTags().getCategoryTag().getKVTag(Tag_Field_Domain_Result_Json);
            CrawlerDomainUrls crawlerDomainUrls = JSON.parseObject(String.valueOf(domainResultJson.getVal()),CrawlerDomainUrls.class);
            String url = crawlerDomainUrls.getUrl();
            Json urlJson = new Json(url);
            String detailUrl= urlJson.jsonPath($_type + ".detailUrl").get();
            String itemId= urlJson.jsonPath($_type + ".itemId").get();
            String userId= urlJson.jsonPath($_type + ".userId").get();
            String taobaoCommentUrl = String.format(commentUrl, itemId, userId, 1);
            httpRequest.setUrl(taobaoCommentUrl);
            httpRequest.getHeaders().put("referer",detailUrl);
            requestRecord.setDownload(true);
            requestRecord.setNeedParsedPage(true);
            requestRecord.setNeedWashPage(true);
            requestRecord.setSkipPipeline(false);
            requestRecord.tagsCreator().bizTags().addCustomKV(Tag_Field_ECommerce_List_Content,url);
            requestRecord.tagsCreator().bizTags().getCategoryTag().getKvTags().remove(Tag_Field_Domain_Result_Json); //移除
        }
        httpRequest.getHeaders().put("user-agent",userAgent);
        httpRequest.getHeaders().put("Cookie",defaultCookie);

        return super.prepareRequest(requestRecord,supportSourceRecords);
    }

    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRecord, HttpPage page) {

        //解析下载数据是否正常，不正常则放入下次循环，不再进行数据清洗
        List<CrawlerRequestRecord> crawlerRequestRecords = new ArrayList<>();

        if (!page.isDownloadSuccess()){
            log.error("TB product comment proxy has error ,will retry");
            addCrawlerRecords(crawlerRequestRecords,crawlerRecord);
            crawlerRecord.setNeedWashPage(false); //下载异常，不需要清洗页面
            return crawlerRequestRecords;
        }

        try {
            Json json = new Json(page.getRawText());
            json = json.removePadding("jsonp_tbcrate_reviews_list");
            int currentPageNum = Integer.valueOf(json.jsonPath($_type + ".currentPageNum").get());
            int maxPage = Integer.valueOf(json.jsonPath($_type + ".maxPage").get());
            List<String> comments = json.jsonPath($_type + ".comments").all();

            if(currentPageNum < maxPage && comments != null && comments.size() > 0){

                String requestUrl = page.getRequest().getUrl();
                String auctionNumId = null;
                String userNumId = null;
                List<NameValuePair> parameters = URLEncodedUtils.parse(requestUrl.split("\\?")[1], Charset.forName("utf-8"));
                for (NameValuePair parameter : parameters) {
                    if ("auctionNumId".equals(parameter.getName())){
                        auctionNumId = parameter.getValue();
                    }
                    if ("userNumId".equals(parameter.getName())){
                        userNumId = parameter.getValue();
                    }
                }
                if (StringUtils.isBlank(auctionNumId) || StringUtils.isBlank(userNumId)){
                    log.error("The current request URL {} is missing important parameters",requestUrl);
                    return null;
                }
                String nextPageUrl = String.format(commentUrl, auctionNumId, userNumId, currentPageNum + 1);
                CrawlerRequestRecord crawlerRequestRecord = CrawlerRequestRecord.builder()
                        .turnPageRequest(crawlerRecord)
                        .httpUrl(nextPageUrl)
                        .needWashed(true)
                        .httpHeads(page.getRequest().getHeaders())
                        .httpHead("user-agent",getOneUserAgent())
                        .releaseTime(System.currentTimeMillis())
                        .copyBizTags()
                        .build();
                crawlerRequestRecords.add(crawlerRequestRecord);


            }
        } catch (Exception e) {
            log.error("taobao product comment download failed,error message {},request detail {}",e.getMessage(),JSON.toJSONString(page.getRequest()));
            addCrawlerRecords(crawlerRequestRecords,crawlerRecord);

            crawlerRecord.setNeedWashPage(false); //下载异常，不需要清洗页面
        }

        return crawlerRequestRecords;
    }

    private void addCrawlerRecords(List<CrawlerRequestRecord> crawlerRequestRecords,CrawlerRequestRecord crawlerRecord){

        int count = 1;
        CrawlerBusinessTags crawlerBusinessTags = crawlerRecord.tagsCreator().bizTags();
        if (crawlerBusinessTags.hasKVTag("taobao_download_retry_count")){
            count = (int) crawlerBusinessTags.getCategoryTag().getKVTag("taobao_download_retry_count").getVal();
            if (count >= 500){
                log.error("taobao product comment download he number of retries exceeds the limit" +
                        ",request url {}",crawlerRecord.getHttpRequest().getUrl());
                return;
            }
        }

        count++;
        crawlerBusinessTags.addCustomKV("taobao_download_retry_count",count);

        CrawlerRequestRecord crawlerRequestRecord = CrawlerRequestRecord.builder()
                .turnPageRequest(crawlerRecord)
                .httpRequest(crawlerRecord.getHttpRequest())
                .httpHead("user-agent",getOneUserAgent())
                .needWashed(true)
                .releaseTime(System.currentTimeMillis())
                .copyBizTags()
                .build();
        crawlerRequestRecord.tagsCreator().requestTags().removeRequestType(CrawlerEnum.CrawlerRequestType.filter);
        crawlerRequestRecords.add(crawlerRequestRecord);
    }

    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerDataList = new ArrayList<>();

        String requestUrl = page.getRequest().getUrl();
        String userNumId = null;
        List<NameValuePair> parameters = URLEncodedUtils.parse(requestUrl.split("\\?")[1], Charset.forName("utf-8"));
        for (NameValuePair parameter : parameters) {
            if ("userNumId".equals(parameter.getName())){
                userNumId = parameter.getValue();
            }
        }

        try {
            Json json = new Json(page.getRawText());
            json = json.removePadding("jsonp_tbcrate_reviews_list");
            List<String> comments = json.jsonPath($_type + ".comments").all();
            if (comments != null && comments.size() > 0){
                for (String comment : comments) {
                    Json commentJson = new Json(comment);
                    String rateId = commentJson.jsonPath($_type + ".rateId").get();
                    String aucNumId = commentJson.jsonPath($_type + ".auction.aucNumId").get();
                    String link = "https:" + commentJson.jsonPath($_type + ".auction.link").get();
                    String date = commentJson.jsonPath($_type + ".date").get();

                    try {
                        long releaseTime = DateUtils.parseDate(date, "yyyy年MM月dd日 HH:mm").getTime();
                        CrawlerData crawlerData = CrawlerData.builder()
                                .data(crawlerRecord, page)
                                .dataId(StringUtils.joinWith("-", crawlerRecord.getDomain(), CrawlerEnum.CrawlerDataType.comment.enumVal(), aucNumId,userNumId,rateId))
                                .parentId(StringUtils.joinWith("-",crawlerRecord.getDomain(),CrawlerEnum.CrawlerDataType.article,aucNumId,userNumId))
                                .url(link)
                                .releaseTime(releaseTime)
                                .content(comment)
                                .requestLabelTag(CrawlerEnum.CrawlerRequestType.result)
                                .requestLabelTag(CrawlerEnum.CrawlerRequestType.filter)
                                .resultLabelTag(CrawlerEnum.CrawlerDataType.comment)
                                .copyBizTags()
                                .build();
                        crawlerData.tagsCreator().bizTags().addDomain(domain());
                        crawlerDataList.add(crawlerData);

                    } catch (ParseException e) {
                        log.info("taobao comment parse date {} is error!",date);
//                        log.error(e.getMessage(),e);
                    }

                }
            } else {
                log.warn("taobo comment request url {} download content is null, detail info {}",page.getRequest().getUrl(),page.getRawText());
            }
        } catch (Exception e) {
            log.error("taobao comment request url {} wash page error: {}",page.getRequest(),e.getMessage());
        }


        return crawlerDataList;
    }

    @Override
    public String domain() {
        return "taobao";
    }

    @Override
    public void initUrlRegulars() {
        addUrlRegular("https://rate.taobao.com/feedRateList.htm\\S*");
    }

    @Override
    public boolean crawlerCheck(CrawlerRequestRecord crawlerRecord) {
        return true;
    }

    @Override
    public void afterExecute(CrawlerRecordContext context) {

    }

    private void initUserAgents( CrawlerRequestRecord supportSourceRecord){

        try {
            HttpPage httpPage = supportSourceRecord.getInternalDownloadPage();
            Json rawText = new Json(httpPage.getRawText());
            String browsers = rawText.jsonPath($_type + ".browsers").get();
            Map<String,Object> map = new Json(browsers).toObject(Map.class);
            for (Map.Entry<String, Object> entry : map.entrySet()) {
                List<String> agents = (List<String>) entry.getValue();
                userAgents.addAll(agents);
            }
        } catch (Exception e) {
            log.error(e.getMessage(),e);
        }
    }

    private String getOneUserAgent(){

        synchronized (lock){
            if (userAgentIndex.get() >= userAgents.size()){
                userAgentIndex.set(0);
            }

            String userAgent = null;
            try {
                userAgent = userAgents.get(userAgentIndex.getAndIncrement());
            } catch (Exception e) {
                log.error(e.getMessage());
            }
            return userAgent;
        }
    }

}
